from NewsMining.db.dbConnect import getDBConnection
from mynewsweb.model import *
from random import shuffle
import html2text
from sqlalchemy import or_

db = getDBConnection()

def getStopWords():
    stopWords = db.query(StopWords.word).all()
    return [i[0] for i in stopWords]

def getData(size):
    positive = db.query(News).filter(News.class_id == 1).all()
    negative = db.query(News).filter(News.class_id == -1).all()
    shuffle(positive)
    shuffle(negative)
    all_news = positive[:size / 2] + negative[:size / 2]
    shuffle(all_news)
    return all_news #positive[:size] + negative[:size]

def getAllData(size, random=False):
    news = db.query(News).filter(News.class_id != 0)

    if random == True:
        shuffle(list(news))
    else:
        news = news.order_by(News.rank)

    return news[:size]

def normalize_content():
    title_unicode_trans = TitleUnicodeTranslate()
    stopWords = getStopWords()
    news = db.query(News).all()

    for n in news:
        print n.news_id
        n.content_normalized = removeStopWords(simpleNormalize(n.content, title_unicode_trans), stopWords)
        db.commit()

def main():
    #normalize_content()
    pass

if __name__ == '__main__':
    main()